#!/usr/bin/env python
#coding=utf-8

import redis
import time
import json

from uniform_encode_decode import uniform_encode, uniform_decode
from CSDN_Extractor import csdn_extract_main
#from CSDN_Items import post_detail_item

r = redis.StrictRedis(host='127.0.0.1', port=6379)

def submit_new_links_to_redis(base_url, new_links_list):
	if not new_links_list or len(new_links_list) <= 0:
		print 'submit_new_links_to_redis no new links'
		return 

	global r
	new_links_redislist_name = 'new_links_redislist'
	for new_url in new_links_list:
		r.lpush(new_links_redislist_name, new_url)	# FIFO, TODO: 实现优先级, JSON

def submit_post_info_to_redis(item_dict):
	if not item_dict or len(item_dict) <= 0:
		return 
	print 'submit_post_info_to_redis'
	#print item_dict
	post_info_redislist_name = 'post_info_redislist'
	json_str = item_dict.toJson()
	print json_str
	global r
	r.lpush(post_info_redislist_name, json_str)

while True:
	data = r.rpop('spider_csdn_data_redislist')
	if not data or len(data) <= 0:
		print None
		time.sleep(0.5)
		continue
	decoded_items = json.loads(data)
	if 'URL' in decoded_items and 'CONTENT' in decoded_items:
		base_url = decoded_items['URL']
		#print 'processing %s' % base_url
		html_doc = uniform_decode(decoded_items['CONTENT'])
		csdn_extract_main(base_url, html_doc, None, submit_new_links_to_redis, submit_post_info_to_redis)

	time.sleep(1)
